package eng;

import java.io.BufferedOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class Spider {
    private static final Log log = LogFactory.getLog(Spider.class);
    
    private static final String puth = "/Users/zhaozhenxing/eng";
    
    public static final String proxyUrl = "proxy.xj.petrochina";
    public static final String proxyPort = "8080";
    
    private static final String webSite = "http://www.kekenet.com";
    
    //	static {
    //		HttpHost proxy = new HttpHost(proxyUrl, Integer.parseInt(proxyPort));
    //		client.getParams().setParameter(ConnRoutePNames.DEFAULT_PROXY, proxy);
    //	}
    
    // 根据课程列表的url获取课程的连接地址
    private List<String> getLessionSrc(String url) throws ClientProtocolException, IOException {
        List<String> list = new ArrayList<String>();
        HttpClient client = new DefaultHttpClient();
        HttpPost post = new HttpPost(url);
        HttpResponse resonse = client.execute(post);
        if (resonse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
            HttpEntity entity = resonse.getEntity();
            if (entity != null) {
                String html = EntityUtils.toString(entity);
                html = new String(html.getBytes("ISO-8859-1"), "gbk");
                Document doc = Jsoup.parseBodyFragment(html, "utf-8");
                String queryStr = "div.c_l_contbox dl dt a[href]";
                Elements elements = doc.select(queryStr);
                if (null != elements && elements.size() > 0) {
                    for (int i = 0; i < elements.size(); i++) {
                        Element element = elements.get(i);
                        String src = element.attr("href");// 每个课程的地址
                        list.add(webSite + src);
                    }
                }
            }
        }
        return list;
    }
    
    // 根据课程的链接获取“下载音频”的连接
    // http://www.kekenet.com/nce/4940.shtml
    // 每个文件的页面地址：http://www.kekenet.com/nce/4947.shtml
    // 下载页面地址：http://u.kekenet.com/index.php?m=content&c=download&articleid=4947
    private String getDownLoadPage(String url) {
        String downLoadPageUrl = "http://u.kekenet.com/index.php?m=content&c=download&articleid=";
        return downLoadPageUrl + url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf("."));
    }
    
    // 根据"下载音频"链接获取“mp3下载地址”连接
    private String getDownLoadUrl(String url) throws ClientProtocolException, IOException {
        HttpClient client = new DefaultHttpClient();
        String downLoadUrl = "";
        HttpPost post = new HttpPost(url);
        HttpResponse resonse = client.execute(post);
        if (resonse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
            HttpEntity entity = resonse.getEntity();
            if (entity != null) {
                String html = EntityUtils.toString(entity);
                // html = new String(html.getBytes("ISO-8859-1"), "gbk");
                Document doc = Jsoup.parseBodyFragment(html, "utf-8");
                // div.c_l_contbox dl dt a[href]
                String queryStr = "div.move script";
                Elements elements = doc.select(queryStr);
                if (null != elements && elements.size() > 0) {
                    Element element = elements.get(4);
                    String js[] = element.html().split(";");
                    downLoadUrl = js[0].substring(js[0].indexOf("'") + 1, js[0].lastIndexOf("'"));
                }
            }
        }
        downLoadUrl = getHDLinkProtectUrl(downLoadUrl);
        return downLoadUrl;
    }
    
    // 获取mp3文件的链接
    private String getHDLinkProtectUrl(String myUrl) {
        String vParamName = "down";
        Date now = new Date();
        String h = String.valueOf(now.getHours());
        h += "";
        if (h.length() < 2) {
            h = "0" + h;
        }
        String m = String.valueOf(now.getMinutes());
        m += "";
        if (m.length() < 2) {
            m = "0" + m;
        }
        String s = String.valueOf(now.getSeconds());
        s += "";
        if (s.length() < 2) {
            s = "0" + s;
        }
        String time = h + m + s;
        String vNewUrl = myUrl + "?" + vParamName + "=" + time;
        return vNewUrl;
    }
    
    // 下载文件
    private synchronized void download(String url, String fileName) throws Exception {
        String file = puth + "/" + fileName;
        BufferedOutputStream out = null;
        byte[] bit = getByte(url);
        if (bit.length > 0) {
            try {
                out = new BufferedOutputStream(new FileOutputStream(file));
                out.write(bit);
                out.flush();
                log.info("Create File success! [" + file + "]");
            } finally {
                if (out != null) {
                    out.close();
                }
            }
        }
    }
    
    private byte[] getByte(String uri) throws Exception {
        HttpClient client = new DefaultHttpClient();
        HttpGet get = new HttpGet(uri);
        try {
            HttpResponse resonse = client.execute(get);
            if (resonse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) {
                HttpEntity entity = resonse.getEntity();
                if (entity != null) {
                    return EntityUtils.toByteArray(entity);
                }
            }
        } catch (Exception e) {
            log.error(e.getMessage());
        } finally {
            client.getConnectionManager().shutdown();
        }
        log.error("获取文件失败！[url=" + uri + "]");
        return new byte[0];
    }
    
    public void getNewEng1() throws Exception {
        //		for (String url : Constans.c01List) {
        //			// 根据列表获取每个文件的页面，此处需要用jsoup解析页面，整理url getLessionSrc
        //			List<String> lessionList = getLessionSrc(url);
        //			// 根据每个文件的页面的url，生成对应下载页面的url。此处只是根据url规则生成下载页面的url; getDownLoadPage
        //			if (null != lessionList && lessionList.size() > 0) {
        //				for (String downLoadPage : lessionList) {
        //					String lessionContentUrl = getDownLoadPage(downLoadPage);
        //					// jsoup解析下载页面，找到下载连接 getDownLoadUrl，
        //					String mp3Url = getDownLoadUrl(lessionContentUrl);
        //					// 下载文件http://d5.kekenet.com/Sound/Article/kekexgn1/1-141.mp3?down=203016
        //					download(
        //							mp3Url,
        //							"c01/"
        //									+ mp3Url.substring(mp3Url.indexOf("-") + 1,
        //											mp3Url.indexOf("?")));
        //				}
        //			}
        //		}
        
        for (String url : Constans.c02List) {
            // 根据列表获取每个文件的页面，此处需要用jsoup解析页面，整理url getLessionSrc
            List<String> lessionList = getLessionSrc(url);
            // 根据每个文件的页面的url，生成对应下载页面的url。此处只是根据url规则生成下载页面的url; getDownLoadPage
            if (null != lessionList && lessionList.size() > 0) {
                for (String downLoadPage : lessionList) {
                    String lessionContentUrl = getDownLoadPage(downLoadPage);
                    // jsoup解析下载页面，找到下载连接 getDownLoadUrl，
                    String mp3Url = getDownLoadUrl(lessionContentUrl);
                    // 下载文件http://d5.kekenet.com/Sound/Article/kekexgn1/1-141.mp3?down=203016
                    download(mp3Url, "c02/" + mp3Url.substring(mp3Url.indexOf("-") + 1, mp3Url.indexOf("?")));
                }
            }
        }
        
        for (String url : Constans.c03List) {
            // 根据列表获取每个文件的页面，此处需要用jsoup解析页面，整理url getLessionSrc
            List<String> lessionList = getLessionSrc(url);
            // 根据每个文件的页面的url，生成对应下载页面的url。此处只是根据url规则生成下载页面的url; getDownLoadPage
            if (null != lessionList && lessionList.size() > 0) {
                for (String downLoadPage : lessionList) {
                    String lessionContentUrl = getDownLoadPage(downLoadPage);
                    // jsoup解析下载页面，找到下载连接 getDownLoadUrl，
                    String mp3Url = getDownLoadUrl(lessionContentUrl);
                    // 下载文件http://d5.kekenet.com/Sound/Article/kekexgn1/1-141.mp3?down=203016
                    download(mp3Url, "c03/" + mp3Url.substring(mp3Url.indexOf("-") + 1, mp3Url.indexOf("?")));
                }
            }
        }
        
        for (String url : Constans.c04List) {
            // 根据列表获取每个文件的页面，此处需要用jsoup解析页面，整理url getLessionSrc
            List<String> lessionList = getLessionSrc(url);
            // 根据每个文件的页面的url，生成对应下载页面的url。此处只是根据url规则生成下载页面的url; getDownLoadPage
            if (null != lessionList && lessionList.size() > 0) {
                for (String downLoadPage : lessionList) {
                    String lessionContentUrl = getDownLoadPage(downLoadPage);
                    // jsoup解析下载页面，找到下载连接 getDownLoadUrl，
                    String mp3Url = getDownLoadUrl(lessionContentUrl);
                    // 下载文件http://d5.kekenet.com/Sound/Article/kekexgn1/1-141.mp3?down=203016
                    download(mp3Url, "c04/" + mp3Url.substring(mp3Url.indexOf("-") + 1, mp3Url.indexOf("?")));
                }
            }
        }
    }
    
    public static void main(String[] args) {
        Spider spider = new Spider();
        try {
            spider.getNewEng1();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    
}
